import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
#from sklearn.ensemble import RandomForestClassifier
#from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.base import TransformerMixin
from sklearn.pipeline import Pipeline, FeatureUnion
from typing import List, Union, Dict
# Warnings will be used to silence various model warnings for tidier output
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
np.random.seed(0)
German_df = pd.read_csv('C:/Users/krish/Downloads/German-reduced.csv')
print(German_df.shape)
print (German_df.columns)
German_df.head()
#feature_list = ['Gender','Age','Marital_Status','NumMonths','Savings_<500','Savings_none','Dependents','Property_rent','Job_management/self-emp/officer/highly qualif emp','Debtors_guarantor','Purpose_CarNew', 'Purpose_furniture/equip','CreditHistory_none/paid','Purpose_CarUsed','CreditAmount','CreditStatus']
feature_list=['Gender','Age','Marital_Status','NumMonths','Savings_<500','Savings_none','Dependents','Property_rent',
'Job_management/self-emp/officer/highly qualif emp','Debtors_guarantor','Purpose_CarNew',
'Purpose_furniture/equip','CreditHistory_none/paid','Purpose_CarUsed','CreditAmount',
'Collateral_real estate','Debtors_none','Job_unemp/unskilled-non resident','Purpose_others',
'CreditHistory_other','PayBackPercent','Collateral_unknown/none','Purpose_education', 'CreditStatus']
X = German_df.iloc[:, :-1]
y = German_df['CreditStatus']
X.head()
y.head()
from imblearn.over_sampling import ADASYN
from collections import Counter
ada = ADASYN(random_state=40)
print('Original dataset shape {}'.format(Counter(y)))
X_res, y_res = ada.fit_resample(X,y)
print('Resampled dataset shape {}'.format(Counter(y_res)))
German_df=X = pd.DataFrame(np.column_stack((X_res, y_res)))
German_df.head()
German_df.columns=feature_list
German_df.head()
from aif360.datasets import GermanDataset
from aif360.metrics import BinaryLabelDatasetMetric
def fair_metrics(fname, dataset, pred, pred_is_dataset=False):
filename = fname
if pred_is_dataset:
dataset_pred = pred
else:
dataset_pred = dataset.copy()
dataset_pred.labels = pred
cols = ['Accuracy', 'F1', 'DI','SPD', 'EOD', 'AOD', 'ERD', 'CNT', 'TI']
obj_fairness = [[1,1,1,0,0,0,0,1,0]]
fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols)
for attr in dataset_pred.protected_attribute_names:
idx = dataset_pred.protected_attribute_names.index(attr)
privileged_groups = [{attr:dataset_pred.privileged_protected_attributes[idx][0]}]
unprivileged_groups = [{attr:dataset_pred.unprivileged_protected_attributes[idx][0]}]
classified_metric = ClassificationMetric(dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred = BinaryLabelDatasetMetric(dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
distortion_metric = SampleDistortionMetric(dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc = classified_metric.accuracy()
f1_sc = 2 * (classified_metric.precision() * classified_metric.recall()) / (classified_metric.precision() + classified_metric.recall())
mt = [acc, f1_sc,
classified_metric.disparate_impact(),
classified_metric.mean_difference(),
classified_metric.equal_opportunity_difference(),
classified_metric.average_odds_difference(),
classified_metric.error_rate_difference(),
metric_pred.consistency(),
classified_metric.theil_index()
]
w_row = []
print('Computing fairness of the model.')
for i in mt:
#print("%.8f"%i)
w_row.append("%.8f"%i)
with open(filename, 'a') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(w_row)
row = pd.DataFrame([mt],
columns = cols,
index = [attr]
)
fair_metrics = fair_metrics.append(row)
fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2)
return fair_metrics
def get_fair_metrics_and_plot(fname, data, model, plot=False, model_aif=False):
pred = model.predict(data).labels if model_aif else model.predict(data.features)
fair = fair_metrics(fname, data, pred)
if plot:
pass
return fair
def get_model_performance(X_test, y_true, y_pred, probs):
accuracy = accuracy_score(y_true, y_pred)
matrix = confusion_matrix(y_true, y_pred)
f1 = f1_score(y_true, y_pred)
return accuracy, matrix, f1
def plot_model_performance(model, X_test, y_true):
y_pred = model.predict(X_test)
probs = model.predict_proba(X_test)
accuracy, matrix, f1 = get_model_performance(X_test, y_true, y_pred, probs)
filename= 'C:/Users/krish/Downloads/filename_mainpjt_results_apr_14_corrected_tgt_no_adasyn_copy.csv'
Since we are dealing with binary label dataset we are using aif360 class BiaryLabelDataset here with target label as CreditStatus and protected attributes as age,gender,marital status. Refer part 11 for more details on protected attributes and privileged classes.
# Fairness metrics
from aif360.metrics import BinaryLabelDatasetMetric
from aif360.explainers import MetricTextExplainer
from aif360.metrics import ClassificationMetric
# Get DF into IBM format
from aif360 import datasets
#converting to aif dataset
aif_dataset = datasets.BinaryLabelDataset(favorable_label = 1, unfavorable_label = 0, df=German_df,
label_names=["CreditStatus"],
protected_attribute_names=["Age","Gender","Marital_Status"],
privileged_protected_attributes = [1,1,1])
#dataset_orig = GermanDataset(protected_attribute_names=['sex'],
# privileged_classes=[[1]],
# features_to_keep=['age', 'sex', 'employment', 'housing', 'savings', 'credit_amount', 'month', 'purpose'],
# custom_preprocessing=custom_preprocessing)
#privileged_groups = [{'Age':1},{'Gender': 1},{'Marital_Status':1}]
#unprivileged_groups = [{'Age':0},{'Gender': 0},{'Marital_Status':0}]
privileged_groups = [{'Gender': 1}]
unprivileged_groups = [{'Gender': 0}]
data_orig_train, data_orig_test = aif_dataset.split([0.7], shuffle=True)
X_train = data_orig_train.features
y_train = data_orig_train.labels.ravel()
X_test = data_orig_test.features
y_test = data_orig_test.labels.ravel()
X_train.shape
X_test.shape
data_orig_test.labels[:10].ravel()
data_orig_train.labels[:10].ravel()
Considering ensemble models for our study.
#Seting the Hyper Parameters
param_grid = {"max_depth": [3,5,7, 10,None],
"n_estimators":[3,5,10,25,50,150],
"max_features": [4,7,15,20]}
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
#Creating the classifier
rf_model = RandomForestClassifier(random_state=40)
grid_search = GridSearchCV(rf_model, param_grid=param_grid, cv=5, scoring='recall', verbose=0)
model = grid_search
mdl_rf = model.fit(data_orig_train.features, data_orig_train.labels.ravel())
from sklearn.metrics import confusion_matrix
conf_mat_rf = confusion_matrix(data_orig_test.labels.ravel(), mdl_rf.predict(data_orig_test.features))
conf_mat_rf
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), mdl_rf.predict(data_orig_test.features)))
unique, counts = np.unique(data_orig_test.labels.ravel(), return_counts=True)
dict(zip(unique, counts))
importances = model.best_estimator_.feature_importances_
indices = np.argsort(importances)
features = data_orig_train.feature_names
#https://stackoverflow.com/questions/48377296/get-feature-importance-from-gridsearchcv
importances
importances[indices]
features
plt.figure(figsize=(20,30))
plt.title('Feature Importances')
plt.barh(range(len(indices)), importances[indices], color='b', align='center')
plt.yticks(range(len(indices)), [features[i] for i in indices])
plt.xlabel('Relative Importance')
plt.show()
import shap
rf_explainer = shap.KernelExplainer(mdl_rf.predict, data_orig_test.features)
rf_shap_values = rf_explainer.shap_values(data_orig_test.features,nsamples=50)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
rf_shap_values
rf_explainer.expected_value
y_test_predict=mdl_rf.predict(data_orig_test.features)
y_test_predict[:12]
data_orig_test.labels[:12].ravel()
data_orig_test.features[:2,:]
y_test_predict.mean()
The explainer expected value is the average model predicted value on input data. Shapely helps to understand how individual features impact the output of each individual instance. The shapely values are model predicted values which may not coincide with actual y test values due to prediction error.
link=”logit” argument converts the logit values to probability
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[0],data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
#https://github.com/slundberg/shap/issues/977
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[0],data_orig_test.features[0],data_orig_test.feature_names)
Features in blue pushes the base value towards lowest values and features in red moves base levels towards higher values.
Shapley values calculate the importance of a feature by comparing what a model predicts with and without the feature. However, since the order in which a model sees features can affect its predictions, this is done in every possible order, so that the features are fairly compared.
The SHAP plot shows features that contribute to pushing the output from the base value (average model output) to the actual predicted value.
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[1], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[1], data_orig_test.features[1],data_orig_test.feature_names)
data_orig_test.feature_names
shap.force_plot(rf_explainer.expected_value,
rf_shap_values, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
p = shap.summary_plot(rf_shap_values, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
Variables with higher impact are displayed at the credit history, credit amount,num of months.
shap.plots._waterfall.waterfall_legacy(rf_explainer.expected_value, rf_shap_values[0],feature_names=data_orig_test.feature_names)
For first instace of input,out of all the displayed variables, CreditHistory with value other is playing major role is pushing the target variable outcome towards predicting 1.
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
Shapley values calculate the importance of a feature by comparing what a model predicts with and without the feature. However, since the order in which a model sees features can affect its predictions, this is done in every possible order, so that the features are fairly compared. https://medium.com/@gabrieltseng/interpreting-complex-models-with-shap-values-1c187db6ec83
shap.plots._waterfall.waterfall_legacy(rf_explainer.expected_value, rf_shap_values[1],feature_names=data_orig_test.feature_names)
For second instace of input,out of all the displayed variables, credit amount is playing major role is pushing the target variable outcome towards predicting 0.
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
perm_rf = PermutationImportance(mdl_rf).fit(data_orig_test.features, data_orig_test.labels.ravel())
perm_imp_1=eli5.show_weights(perm_rf,feature_names = data_orig_test.feature_names)
perm_imp_1
plt.show()
eli5 provides a way to compute feature importances for any black-box estimator by measuring how score decreases when a feature is not available; the method is also known as “permutation importance” or “Mean Decrease Accuracy (MDA)”.
The first number in each row shows how much model performance decreased with a random shuffling (in this case, using "accuracy" as the performance metric).
Like most things in data science, there is some randomness to the exact performance change from a shuffling a column. We measure the amount of randomness in our permutation importance calculation by repeating the process with multiple shuffles. The number after the ± measures how performance varied from one-reshuffling to the next.
You'll occasionally see negative values for permutation importances. In those cases, the predictions on the shuffled (or noisy) data happened to be more accurate than the real data. This happens when the feature didn't matter (should have had an importance close to 0), but random chance caused the predictions on shuffled data to be more accurate. This is more common with small datasets, like the one in this example, because there is more room for luck/chance.
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_rf, X_test, y_test)
fair = get_fair_metrics_and_plot(filename, data_orig_test, mdl_rf)
fair
type(data_orig_train)
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train = RW.fit_transform(data_orig_train)
data_transf_test = RW.transform(data_orig_test)
fair_rw = get_fair_metrics_and_plot(filename, data_transf_test, mdl_rf, plot=False)
fair_rw
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train = DIR.fit_transform(data_orig_train)
# Train and save the model
#rf_transf = model.fit(data_transf_train.features,data_transf_train.labels.ravel())
fair_dir = get_fair_metrics_and_plot(filename, data_orig_test, mdl_rf, plot=False)
fair_dir
#!pip install --user --upgrade tensorflow==1.15.0
#2.2.0
#!pip uninstall tensorflow
#!pip install "tensorflow==1.15"
#!pip install --upgrade tensorflow-hub
#%tensorflow_version 1.15
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope1',reuse=tf.AUTO_REUSE) as scope:
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model.fit(data_orig_train)
fair_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_ad
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model = PrejudiceRemover()
# Train and save the model
debiased_model.fit(data_orig_train)
fair_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_pr
y_pred = debiased_model.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_rf.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_rf.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP = EOPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = EOPP.predict(data_orig_test_pred)
fair_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_eo
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=42)
CPP = CPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = CPP.predict(data_orig_test_pred)
fair_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_ceo
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC = ROC.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = ROC.predict(data_orig_test_pred)
fair_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
fair_roc
from xgboost import XGBClassifier
estimator = XGBClassifier(seed=40)
parameters = {
'max_depth': range (2, 10, 2),
'n_estimators': range(60, 240, 40),
'learning_rate': [0.1, 0.01, 0.05]
}
grid_search = GridSearchCV(
estimator=estimator,
param_grid=parameters,
scoring = 'recall',
cv = 5,
verbose=0
)
model_xg=grid_search
mdl_xgb = model_xg.fit(data_orig_train.features, data_orig_train.labels.ravel())
importances_xg = model_xg.best_estimator_.feature_importances_
indices_xg = np.argsort(importances_xg)
features = data_orig_train.feature_names
#https://stackoverflow.com/questions/48377296/get-feature-importance-from-gridsearchcv
importances_xg
importances_xg[indices_xg]
features
plt.figure(figsize=(20,30))
plt.title('Feature Importances')
plt.barh(range(len(indices_xg)), importances_xg[indices_xg], color='b', align='center')
plt.yticks(range(len(indices_xg)), [features[i] for i in indices_xg])
plt.xlabel('Relative Importance')
plt.show()
import shap
xg_shap_values_t1 = shap.KernelExplainer(mdl_xgb.predict,data_orig_train.features)
xgb_explainer = shap.KernelExplainer(mdl_xgb.predict, data_orig_test.features)
xgb_shap_values = xgb_explainer.shap_values(data_orig_test.features,nsamples=10)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
xgb_shap_values
shap.initjs()
shap.force_plot(xgb_explainer.expected_value,xgb_shap_values[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
shap.initjs()
shap.force_plot(xgb_explainer.expected_value,xgb_shap_values[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
shap.force_plot(xgb_explainer.expected_value,
xgb_shap_values, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
p = shap.summary_plot(xgb_shap_values, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
The variables with higher impact are the ones in the top age,gender,marital status.
shap.plots._waterfall.waterfall_legacy(xgb_explainer.expected_value, xgb_shap_values[0,:],feature_names=data_orig_test.feature_names)
Here credit history other and age are moving target outcome towards right i.e., 1.
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
shap.plots._waterfall.waterfall_legacy(xgb_explainer.expected_value, xgb_shap_values[1],feature_names=data_orig_test.feature_names)
Here Credit Amount is moving the target result towards zero.
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
perm_xgb = PermutationImportance(mdl_xgb).fit(data_orig_test.features, data_orig_test.labels.ravel())
perm_imp_2=eli5.show_weights(perm_xgb,feature_names = data_orig_test.feature_names)
perm_imp_2
plt.show()
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_xgb, X_test, y_test)
fair = get_fair_metrics_and_plot(filename, data_orig_test, mdl_xgb)
fair
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train = RW.fit_transform(data_orig_train)
data_transf_test = RW.transform(data_orig_test)
fair_rw = get_fair_metrics_and_plot(filename, data_transf_test, mdl_xgb, plot=False)
fair_rw
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train = DIR.fit_transform(data_orig_train)
fair_dir = get_fair_metrics_and_plot(filename, data_orig_test, mdl_xgb, plot=False)
fair_dir
#!pip install tensorflow
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope2',reuse=tf.AUTO_REUSE) as scope:
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model.fit(data_orig_train)
fair_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_ad
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model = PrejudiceRemover()
# Train and save the model
debiased_model.fit(data_orig_train)
fair_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_pr
y_pred = debiased_model.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_xgb.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_xgb.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP = EOPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = EOPP.predict(data_orig_test_pred)
fair_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_eo
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=40)
CPP = CPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = CPP.predict(data_orig_test_pred)
fair_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_ceo
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC = ROC.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = ROC.predict(data_orig_test_pred)
fair_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
fair_roc
from xgboost import XGBClassifier
model_xgb2 = XGBClassifier(seed=40)
mdl_xgb2 = model_xgb2.fit(data_orig_train.features, data_orig_train.labels.ravel())
importances_xg2 = model_xgb2.feature_importances_
indices_xg2 = np.argsort(importances_xg2)
features2 = data_orig_train.feature_names
#https://stackoverflow.com/questions/48377296/get-feature-importance-from-gridsearchcv
importances_xg2
importances_xg2[indices_xg2]
features2
plt.figure(figsize=(20,30))
plt.title('Feature Importances')
plt.barh(range(len(indices_xg2)), importances_xg2[indices_xg2], color='b', align='center')
plt.yticks(range(len(indices_xg2)), [features2[i] for i in indices_xg2])
plt.xlabel('Relative Importance')
plt.show()
import shap
xg_shap_values_t = shap.KernelExplainer(mdl_xgb2.predict,data_orig_train.features)
xgb_explainer2 = shap.KernelExplainer(mdl_xgb2.predict, data_orig_test.features)
xgb_shap_values2 = xgb_explainer2.shap_values(data_orig_test.features,nsamples=10)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
xgb_shap_values2
shap.initjs()
shap.force_plot(xgb_explainer2.expected_value,xgb_shap_values2[0:,], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
The field age is pushing target outcome towards lower value and collateral unknown/none, purpose others are pushing the target towards higher value which resulted in the final probability of occurrance as .73
shap.initjs()
shap.force_plot(xgb_explainer2.expected_value,xgb_shap_values2[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Here only credit amount has impact in moving target outcome towards lower value than the base value which is the mean value to target outcome
data_orig_test.feature_names
shap.force_plot(xgb_explainer2.expected_value,
xgb_shap_values2, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
p = shap.summary_plot(xgb_shap_values2, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
variables with higher impact are CreditAmount,NumMonths,Savings.
shap.plots._waterfall.waterfall_legacy(xgb_explainer2.expected_value, xgb_shap_values2[0,:],feature_names=data_orig_test.feature_names)
Here purpose other and collateral unknown/none are pushing target to higher value and age is pushing it towards lower value.
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
shap.plots._waterfall.waterfall_legacy(xgb_explainer2.expected_value, xgb_shap_values2[1],feature_names=data_orig_test.feature_names)
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
perm_xgb2 = PermutationImportance(mdl_xgb2).fit(data_orig_test.features, data_orig_test.labels.ravel())
perm_imp_3=eli5.show_weights(perm_xgb2,feature_names = data_orig_test.feature_names)
perm_imp_3
plt.show()
from eli5 import show_prediction
show_prediction(mdl_xgb2, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_xgb2, X_test, y_test)
fair = get_fair_metrics_and_plot(filename, data_orig_test, mdl_xgb2)
fair
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train = RW.fit_transform(data_orig_train)
data_transf_test = RW.transform(data_orig_test)
fair_rw = get_fair_metrics_and_plot(filename, data_transf_test, mdl_xgb2, plot=False)
fair_rw
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train = DIR.fit_transform(data_orig_train)
fair_dir = get_fair_metrics_and_plot(filename, data_orig_test, mdl_xgb2, plot=False)
fair_dir
#!pip install tensorflow
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope3',reuse=tf.AUTO_REUSE) as scope:
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model.fit(data_orig_train)
fair_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_ad
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model = PrejudiceRemover()
# Train and save the model
debiased_model.fit(data_orig_train)
fair_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_pr
y_pred = debiased_model.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_xgb.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_xgb.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP = EOPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = EOPP.predict(data_orig_test_pred)
fair_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_eo
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=40)
CPP = CPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = CPP.predict(data_orig_test_pred)
fair_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_ceo
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC = ROC.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = ROC.predict(data_orig_test_pred)
fair_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
fair_roc
#Creating the classifier
rf_model2 = RandomForestClassifier(random_state=40)
model=rf_model2
mdl_rf2 = model.fit(data_orig_train.features, data_orig_train.labels.ravel())
from sklearn.metrics import confusion_matrix
conf_mat_rf = confusion_matrix(data_orig_test.labels.ravel(), mdl_rf2.predict(data_orig_test.features))
conf_mat_rf
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), mdl_rf2.predict(data_orig_test.features)))
unique, counts = np.unique(data_orig_test.labels.ravel(), return_counts=True)
dict(zip(unique, counts))
import shap
rf_shap_values_t2 = shap.KernelExplainer(mdl_rf2.predict,data_orig_train.features)
rf_explainer2 = shap.KernelExplainer(mdl_rf2.predict, data_orig_test.features)
rf_shap_values2 = rf_explainer2.shap_values(data_orig_test.features,nsamples=10)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
rf_shap_values2
rf_explainer2.expected_value
rf_shap_values2
shap.initjs()
shap.force_plot(rf_explainer2.expected_value,rf_shap_values2[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
shap.initjs()
shap.force_plot(rf_explainer2.expected_value,rf_shap_values2[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
shap.initjs()
shap.force_plot(rf_explainer2.expected_value,rf_shap_values2[2,:], data_orig_test.features[2],data_orig_test.feature_names,link='logit')
data_orig_test.feature_names
shap.force_plot(rf_explainer2.expected_value,
rf_shap_values2, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
p = shap.summary_plot(rf_shap_values2, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
Variables with higher impact are displayed at the top such as gender,age,nummonths etc
shap.plots._waterfall.waterfall_legacy(rf_explainer2.expected_value, rf_shap_values2[0,:],feature_names=data_orig_test.feature_names)
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
shap.plots._waterfall.waterfall_legacy(rf_explainer2.expected_value, rf_shap_values2[1],feature_names=data_orig_test.feature_names)
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
perm_rf2 = PermutationImportance(mdl_rf2).fit(data_orig_test.features, data_orig_test.labels.ravel())
data_orig_test.labels[:10,:].ravel()
perm_imp_11=eli5.show_weights(perm_rf2,feature_names = data_orig_test.feature_names)
perm_imp_11
plt.show()
show_prediction(mdl_rf2, data_orig_test.features[0], show_feature_values=True,feature_names = data_orig_test.feature_names)
from eli5 import show_prediction
show_prediction(mdl_rf2, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_rf2, X_test, y_test)
fair = get_fair_metrics_and_plot(filename, data_orig_test, mdl_rf2)
fair
type(data_orig_train)
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train = RW.fit_transform(data_orig_train)
data_transf_test = RW.transform(data_orig_test)
fair_rw = get_fair_metrics_and_plot(filename, data_transf_test, mdl_rf2, plot=False)
fair_rw
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train = DIR.fit_transform(data_orig_train)
# Train and save the model
#rf_transf = model.fit(data_transf_train.features,data_transf_train.labels.ravel())
fair_dir = get_fair_metrics_and_plot(filename, data_orig_test, mdl_rf2, plot=False)
fair_dir
#!pip install --user --upgrade tensorflow==1.15.0
#2.2.0
#!pip uninstall tensorflow
#!pip install "tensorflow==1.15"
#!pip install --upgrade tensorflow-hub
#%tensorflow_version 1.15
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope1',reuse=tf.AUTO_REUSE) as scope:
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model.fit(data_orig_train)
fair_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_ad
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model = PrejudiceRemover()
# Train and save the model
debiased_model.fit(data_orig_train)
fair_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_pr
y_pred = debiased_model.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_rf2.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_rf2.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP = EOPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = EOPP.predict(data_orig_test_pred)
fair_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_eo
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=42)
CPP = CPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = CPP.predict(data_orig_test_pred)
fair_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_ceo
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC = ROC.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = ROC.predict(data_orig_test_pred)
fair_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
fair_roc
from sklearn import neighbors
n_neighbors = 15
knn = neighbors.KNeighborsClassifier(n_neighbors, weights='distance')
knn.fit(data_orig_train.features, data_orig_train.labels.ravel())
knn_explainer = shap.KernelExplainer(knn.predict, data_orig_test.features)
knn_shap_values = knn_explainer.shap_values(data_orig_test.features,nsamples=10)
#shap.dependence_plot(0, knn_shap_values, data_orig_test.features)
# plot the SHAP values for the 0th observation
shap.force_plot(knn_explainer.expected_value,knn_shap_values[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
# plot the SHAP values for the 1st observation
shap.force_plot(knn_explainer.expected_value,knn_shap_values[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
shap.force_plot(knn_explainer.expected_value, knn_shap_values, data_orig_test.feature_names,link='logit')
shap.summary_plot(knn_shap_values, data_orig_test.features,feature_names=data_orig_test.feature_names, plot_type="violin")
Feature Importance
perm_imp_11=eli5.show_weights(knn,feature_names = data_orig_test.feature_names) perm_imp_11 plt.show()
from eli5 import show_prediction
show_prediction(mdl_rf2, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(knn, X_test, y_test)
fair = get_fair_metrics_and_plot(filename, data_orig_test, knn)
fair
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train = RW.fit_transform(data_orig_train)
data_transf_test = RW.transform(data_orig_test)
fair_rw = get_fair_metrics_and_plot(filename, data_transf_test, knn, plot=False)
fair_rw
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train = DIR.fit_transform(data_orig_train)
fair_dir = get_fair_metrics_and_plot(filename, data_orig_test, knn, plot=False)
fair_dir
#!pip install tensorflow
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope4',reuse=tf.AUTO_REUSE) as scope:
debiased_model = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model.fit(data_orig_train)
fair_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_ad
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model = PrejudiceRemover()
# Train and save the model
debiased_model.fit(data_orig_train)
fair_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model, plot=False, model_aif=True)
fair_pr
y_pred = debiased_model.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = knn.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = knn.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP = EOPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = EOPP.predict(data_orig_test_pred)
fair_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_eo
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=40)
CPP = CPP.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = CPP.predict(data_orig_test_pred)
fair_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
fair_ceo
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC = ROC.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred = ROC.predict(data_orig_test_pred)
fair_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
fair_roc